# from tqdm import tqdm
# import json
# import re
# import numpy as np
# import os
# import glob
# # from fuzzywuzzy import fuzz, process
#
# def gen_opcode_hist():
#     # load opcodes
#     opcodes = json.load(open('opcodes.json'))
#
#     opc_contracts = [(opcode, opcodes[opcode]['contracts']) for opcode in opcodes]
#     opc_contracts = sorted(opc_contracts, key = lambda k: k[1], reverse=True)
#
#     opcodes = [(opcode, opcodes[opcode]['freq']) for opcode in opcodes]
#     opcodes = sorted(opcodes, key = lambda k: k[1], reverse=True)
#
#     print "length: %d" % len(opcodes)
#
#     with open('opcodehist.dat', 'w') as of:
#         of.write('opcode frequency\n')
#         for i in xrange(0,25):
#             of.write('%s %d\n' % (opcodes[i][0], opcodes[i][1]))
#         of.flush()
#         of.close()
#
#     with open('opcodecont.dat', 'w') as of:
#         of.write('opcode contracts\n')
#         for i in xrange(0,25):
#             of.write('%s %d\n' % (opc_contracts[i][0], opc_contracts[i][1]))
#         of.flush()
#         of.close()
#
# def gen_contract_hist():
#     contracts = json.load(open('contracts.json'))
#     oplengths = [contracts[contract]['oplength'] for contract in contracts]
#     oplengths = sorted(oplengths, reverse=True)
#
#     print "Max: %d, Min: %d, Mean: %d, Median: %d" % (max(oplengths), min(oplengths), np.mean(oplengths), np.median(oplengths))
#
#     with open('contract_length.dat', 'w') as of:
#         for oplength in oplengths:
#             of.write('%d\n' % oplength)
#         of.flush()
#         of.close()
#
# def process_worth_str(istr):
#     tag_pattern = r"<.+?>"
#     outstr = re.sub(tag_pattern, "", istr).lower()
#     # It's wei or ether
#     # factor = 1
#     # if(outstr.find('wei') > 0):
#     #     factor = 1000000000000000000
#     factor = 1000000000000000000
#     if(outstr.find('wei') > 0):
#         factor = 1
#     outstr = re.sub("ether", "", outstr)
#     outstr = re.sub("wei", "", outstr)
#     outstr = re.sub(",", "", outstr)
#     outval = float(outstr)
#     return outval * factor
#
# def gen_txnesting_hist(tfile):
#     nesting = [(transaction['txid'],len(transaction['transactions'])) for transaction in tqdm(tfile)]
#     nesting = sorted(nesting, key = lambda k: k[1], reverse=True)
#     nestingvals = [element[1] for element in nesting]
#     print "Writing..."
#     with open('txnesting.csv', 'w') as of:
#         for element in tqdm(nestingvals):
#             of.write('%d\n' % element)
#         of.flush()
#         of.close()
#
#     hist = np.histogram()
#
#     # with open('txnesting.dat','w') as of:
#     #     of.write('txid nesting\n')
#     #     for element in nesting:
#     #         of.write('%s %d\n' % (element[0], element[1]))
#     #     of.flush()
#     #     of.close()
#
# def gen_txworth_hist(tfile):
#     txworth = [(transaction['txid'], process_worth_str(transaction['transactions'][0]['worth'])) for transaction in tqdm(tfile)]
#     txworth = sorted(txworth, key = lambda k : k[1], reverse=True)
#     txvalues = [element[1] for element in txworth]
#     print "Transaction value - min:%f, max:%f, mean:%f, median:%f" % (np.min(txvalues), np.max(txvalues), np.mean(txvalues), np.median(txvalues))
#     with open('txvalueslog.csv', 'w') as of:
#         i=0
#         for element in txworth:
#             if(element[1] > 0):
#                 i+=1
#                 of.write('%f\n' % np.log(element[1]))
#         of.flush()
#         of.close()
#     print "Wrote %d elements." % i
#     # with open('txworth.dat', 'w') as of:
#     #     of.write('txid worth\n')
#     #     for element in txworth:
#     #         of.write('%s %f\n' % (element[0], element[1]))
#     #     of.flush()
#     #     of.close()
#     # with open('txworth_nozero_log.dat', 'w') as of:
#     #     of.write('txid worth\n')
#     #     for element in txworth:
#     #         if element[1] > 0: of.write('%s %f\n' % (element[0], log(element[1])))
#     #     of.flush()
#     #     of.close()
#     # with open('txworth_nozero.dat', 'w') as of:
#     #     of.write('txid worth\n')
#     #     for element in txworth:
#     #         if element[1] > 0: of.write('%s %f\n' % (element[0], element[1]))
#     #     of.flush()
#     #     of.close()
#
# def gen_contract_tx(tfile):
#     # Build unique transaction count for contracts
#     txcount = {}
#     contract_balance = {}
#     print "Collecting transaction count..."
#     for transaction in tqdm(tfile):
#         for tx in transaction['transactions']:
#             if 'from' in tx:
#                 if tx['from'] not in txcount:
#                     txcount[tx['from']] = 0
#                 txcount[tx['from']] += 1
#             if 'to' in tx:
#                 if tx['to'] not in txcount:
#                     txcount[tx['to']] = 0
#                 txcount[tx['to']] += 1
#     cbalancefile = json.load(open('../contracts/contract_data/contract_balance.json'))
#     cfile = json.load(open('contracts.json'))
#     transactions = []
#     print "Filtering..."
#     for contract in tqdm(cfile):
#         if contract in txcount:
#             transactions.append(txcount[contract])
#         else:
#             transactions.append(0)
#     # transactions = sorted(transactions, reverse=True)
#     downscale_binsize = 100
#     dstxs = []
#     print "Downscaling..."
#     for i in tqdm(xrange(0, len(transactions), downscale_binsize)):
#         dstxs.append(np.mean(transactions[i:(i+downscale_binsize)]))
#     transactions = dstxs
#     print "Writing..."
#     with open('ctxcount.csv', 'w') as of:
#         for tcount in transactions:
#             of.write('%f\n' % tcount)
#         of.flush()
#         of.close()
#     print "loading balances.."
#     balances = []
#     weiconst = 1000000000000000000
#     for contract in tqdm(cfile):
#         if contract in cbalancefile:
#             balances.append(cbalancefile[contract]/weiconst)
#         else:
#             balances.append(0)
#
#     print "Min Balance: %d, Max Balance: %d, Sum: %f, Mean Balance: %f, Median Balance: %f" % (np.min(balances), np.max(balances), np.sum(balances), np.mean(balances), np.median(balances))
#     balances = sorted(balances, reverse=True)
#     dsbalances = []
#     print "Downscaling..."
#     for i in tqdm(xrange(0, len(balances), downscale_binsize)):
#         dsbalances.append(np.mean(balances[i:(i+downscale_binsize)]))
#     balances = dsbalances
#
#     with open('cbalance.csv', 'w') as of:
#         for tbalance in balances:
#             of.write('%f\n' % tbalance)
#         of.flush()
#         of.close()
#     print "Done."
#
# def load_report(filen, contract_name, contract_sats, failed):
#     try:
#         with open(filen) as f:
#             explored_paths = int(f.readline())
#             pathno = int(re.findall(r"number of path: ([\d]+)", f.readline())[0])
#             f.readline()
#             f.readline()
#             concurrency_pairs = int(f.readline())
#             f.readline()
#             timestamp_dependent = False
#             td = f.readline()
#             if td[0:2] == "no":
#                 timestamp_dependent = False
#             elif td[0:3] == "yes":
#                 timestamp_dependent = True
#             else:
#                 failed.append(filen)
#                 return
#             exec_time = 0
#             try:
#                 exec_time = float(f.readline())
#             except:
#                 failed.append(filen)
#                 return
#             contract_sats.append((contract_name, explored_paths, pathno, concurrency_pairs, timestamp_dependent, exec_time))
#     except ValueError:
#         return
#         print "ValueError in file %s. contents - " % filen
#         with open(filen) as f:
#             print f.read()
#
# def load_sat_stats():
#     dirname = "stats"
#     files = os.listdir(dirname)
#     pattern = r"(0x[a-f\d]+)\."
#     contract_sats = []
#     failed = []
#     for filen in tqdm(files):
#         if filen.endswith(".report"):
#             contract_name = re.findall(pattern,filen)[0]
#             load_report(dirname + "/" + filen, contract_name, contract_sats, failed)
#     print "Loading contract db..."
#     contracts = json.loads(open('contracts.json').read())
#     stats = []
#     for entry in contract_sats:
#         if entry[0] in contracts:
#             stats.append((entry[1], entry[5]))
#     stats = sorted(stats, key = lambda k : k[0])
#     print "Writing..."
#     with open('running-time.csv', 'w') as rtcsv:
#         rtcsv.write("contract,paths,time\n")
#         for i in tqdm(xrange(0, len(stats))):
#             rtcsv.write("%d,%d,%f\n" % (i, stats[i][0], stats[i][1]))
#     print "Done."
#     with open('sat_stats.json','w') as of:
#         of.write(json.dumps(contract_sats, indent=1))
#     transaction_race = 0
#     timestamp = 0
#     paths = []
#     for entry in contract_sats:
#         paths.append(entry[1])
#         if entry[3] > 0: transaction_race+=1
#         if entry[4] == True: timestamp += 1
#     print "Average number of paths: %d" % np.mean(paths)
#     print "Transaction Race: %d" % transaction_race
#     print "TimeStamp Dependence: %d" % timestamp
#     print "Average time: %f" % np.mean([entry[5] for entry in contract_sats])
#
# # load_sat_stats()
#
# def check_unique():
#     # Load all contracts
#     cpath = "../contracts/contract_data"
#     cfiles = os.listdir(cpath)
#     contracts = {}
#     cmains = {}
#
#     for cfile in tqdm(cfiles):
#         if not cfile.endswith('.json'): continue
#         if cfile == "contract_balance.json": continue
#         cjson = json.loads(open(cpath+'/'+cfile).read())
#         contracts.update(cjson)
#         for contract in cjson:
#             cmains[contract] = cjson[contract][1]
#
#     print "Loaded %d contracts." % len(cmains)
#     print "Removing duplicates..."
#
#     original_cmains = {}
#
#     for element in tqdm(cmains):
#         found = False
#         for o in original_cmains:
#             if cmains[element] == original_cmains[o]:
#                 found = True
#                 break
#         if not found:
#             original_cmains[element] = cmains[element]
#
#     print "%d original contracts." % len(original_cmains)
#
#     with open('unique_contracts.json', 'w') as of:
#         of.write(json.dumps(original_cmains, indent=1))
#         of.flush()
#         of.close()
#
#     print "merging..."
#
#     duplicates = []
#
#     for contract in tqdm(cmains):
#         found = False
#         for l in duplicates:
#             if cmains[l[0]] == cmains[contract]:
#                 l.append(contract)
#                 found = True
#                 break
#         if not found:
#             duplicates.append([contract])
#
#     print "%d elements in merge list." % len(duplicates)
#
#     with open('duplicate_contracts.json', 'w') as of:
#         of.write(json.dumps(duplicates, indent=1))
#         of.flush()
#         of.close()
#
# # check_unique()
#
# def get_all():
#     sats = json.loads(open('sat_stats.json').read())
#     cstack = json.loads(open('callstack_stats.json').read())
#     reentrancy = json.loads(open('re_stats.json').read())
#
#     all_problematic = []
#
#     for c in cstack:
#         all_problematic.append(c)
#
#     # all_problematic = cstack
#     t_race = []
#     ts_depend = []
#     for entry in sats:
#         if entry[0] in all_problematic: continue
#         if entry[3] > 0:
#             # if entry[0] == '0xbb9bc244d798123fde783fcc1c72d3bb8c189413': print "first"
#             t_race.append(entry[0])
#             all_problematic.append(entry[0])
#         if entry[4] == True:
#             # if entry[0] == '0xbb9bc244d798123fde783fcc1c72d3bb8c189413': print "second"
#             ts_depend.append(entry[0])
#             if entry[0] not in all_problematic: all_problematic.append(entry[0])
#
#     old_prob = all_problematic[:]
#
#     for c in reentrancy:
#         all_problematic.append(c)
#
#     print "%d transaction race, %d timestamp, %d in cstack, %d in reentrancy." % (len(t_race), len(ts_depend), len(cstack), len(reentrancy))
#
#     print "%d problematic contracts" % len(all_problematic)
#     print "Loading contract bytecode..."
#     # cpath = "../contracts/contract_data"
#     # cfiles = os.listdir(cpath)
#
#     cmains = {}
#
#     cfiles = glob.glob('../contracts/contract_data/contract*.json')
#
#     for cfile in tqdm(cfiles):
#         # if not cfile.endswith('.json'): continue
#         # if cfile == "contract_balance.json": continue
#         cjson = json.loads(open(cfile).read())
#         for contract in all_problematic:
#             if contract in cmains: continue
#             if contract in cjson:
#                 cmains[contract] = cjson[contract][1]
#
#     # Remove duplicates
#     cmains_sorted = {}
#     # for c in cmains:
#     #     cmains_sorted.append((c, cmains[c]))
#
#     # cmains_sorted = sorted(cmains_sorted, lamda k: k[1])
#
#     threshold = 99
#
#     print "Removing duplicates..."
#
#     duplicates = 0
#
#     new_t_race = []
#     new_ts_depend = []
#     new_cstack = []
#     new_reentrancy = []
#
#     print "Transaction Race..."
#     for c in tqdm(t_race):
#         found = False
#         for cont in new_t_race:
#             if cmains[cont] == cmains[c]:
#                 found = True
#                 break
#         if not found:
#             new_t_race.append(c)
#
#     print "Timestamp..."
#     for c in tqdm(ts_depend):
#         found = False
#         for cont in new_ts_depend:
#             if cmains[cont] == cmains[c]:
#                 found = True
#                 break
#         if not found:
#             new_ts_depend.append(c)
#
#     print "Callstack..."
#     for c in tqdm(cstack):
#         found = False
#         for cont in new_cstack:
#             if cmains[cont] == cmains[c]:
#                 found = True
#                 break
#         if not found:
#             new_cstack.append(c)
#
#     print "Reentrancy..."
#     for c in tqdm(reentrancy):
#         found = False
#         for cont in new_reentrancy:
#             if cmains[cont] == cmains[c]:
#                 found = True
#                 break
#         if not found:
#             new_reentrancy.append(c)
#
#     print "All..."
#     for contract in tqdm(cmains):
#         found = False
#         for c in cmains_sorted:
#             # if fuzz.ratio(cmains_sorted[c], cmains[contract]) >= threshold:
#             if cmains[contract] == cmains_sorted[c]:
#                 found = True
#                 break
#         if not found:
#             cmains_sorted[contract] = cmains[contract]
#         else:
#             duplicates += 1
#             # print "duplicate count: %d, original len: %d" % (duplicates, len(cmains_sorted))
#
#     print "Original Contracts: %d" % len(cmains_sorted)
#
#     print "Original Counts - "
#     print "CallStack - %d, Transaction Race - %d, Timestamp Dependence - %d, Reentrancy - %d" % (len(new_cstack), len(new_t_race), len(new_ts_depend), len(new_reentrancy))
#
#     return old_prob
#
#     # return cmains_sorted
# # 3068654.2186862295
# # 3067825.0
# # cmains = get_all()
#
# def get_source_timestamp():
#     sats = json.loads(open('sat_stats.json').read())
#     source_list = []
#
#     for entry in sats:
#         if entry[4] == True:
#             source_list.append(entry[0])
#
#     from get_source import get_contract_code
#
#     for c in tqdm(source_list):
#         code = get_contract_code(c)
#
#         if len(code[1]) > 0:
#             with open('timestamp_source/'+c+'.sol', 'w') as of:
#                 of.write(code[1][0])
#                 of.flush()
#                 of.close()
#
# def get_tsdepend_timestamp():
#     sats = json.loads(open('sat_stats.json').read())
#     source_list = []
#
#     for entry in sats:
#         if entry[3] > 0:
#             source_list.append(entry[0])
#
#     from get_source import get_contract_code
#
#     for c in tqdm(source_list):
#         code = get_contract_code(c)
#
#         if len(code[1]) > 0:
#             with open('tod_source/'+c+'.sol', 'w') as of:
#                 of.write(code[1][0])
#                 of.flush()
#                 of.close()
#
# def get_source_stats():
#     contracts = []
#
#     cpath = "../contracts/contract_data"
#     cfiles = os.listdir(cpath)
#
#     print "Loading contracts..."
#
#     for cfile in tqdm(cfiles):
#         if not cfile.endswith('.json'): continue
#         if cfile == "contract_balance.json": continue
#         cjson = json.loads(open(cpath+'/'+cfile).read())
#         for contract in cjson: contracts.append(contract)
#
#     print "Checking for source..."
#
#     from get_source import get_contract_code
#
#     with_source = 0
#
#     for contract in tqdm(contracts):
#         if(len(get_contract_code(contract)[1]) > 0): with_source+=1
#
#     print "%d contracts have source out of %d." % (with_source, len(contracts))
#
# # get_source_stats()
#
# get_all()
#
# # get_tsdepend_timestamp()
#
# # print "Loading transactions..."
# # tfile = json.loads(open('transactions.json').read())
# # print "Generating contract txcount and balances..."
# # gen_contract_tx(tfile)
#
# # print "Generating Contract oplength..."
# # gen_contract_hist()
# # print "Generating Opcode histogram..."
# # gen_opcode_hist()
# # print "Generating Transaction Nesting Histogram..."
# # gen_txnesting_hist(tfile)
# # print "Generating Transaction worth histogram..."
# # gen_txworth_hist(tfile)